import datetime
print(datetime.datetime.now())
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
import seaborn as sns
from sklearn.metrics import silhouette_score, silhouette_samples
import sklearn.metrics
from sklearn.preprocessing import StandardScaler
from sklearn.cluster import KMeans, DBSCAN, AgglomerativeClustering
from sklearn.mixture import GaussianMixture
import itertools
import scipy
from IPython.core.interactiveshell import InteractiveShell
InteractiveShell.ast_node_interactivity = "all"
# This will ensure that matplotlib figures don't get cut off when saving with savefig()
#from matplotlib import rcParams
#rcParams.update({'figure.autolayout': True})
# Bank Marketing Data
#df = pd.read_csv("https://raw.githubusercontent.com/stepthom/sandbox/master/data/bank-additional/bank-additional-full.csv")
#df = df.rename(index=str, columns={"y": "bought"})
# Default dataset from ISLR
#df = pd.read_csv("https://raw.githubusercontent.com/stepthom/sandbox/master/data/islr_credit.csv")
#df = df.drop(["Unnamed: 0"], axis=1)
For educational purposes, we'll generate a synthetic dataset, rather than using a real one, at first. We'll create a dataset that has two features, both informative, with some overlap, but not much.
Later, we'll graduate to using real-world datasets with more features and less seperation between classes.
from sklearn.datasets import make_classification
import random
# After experiementation, this random state generates a "good looking" dataset
r = 4184
X, y = make_classification(n_samples=500, n_features=2, n_redundant=0, n_informative=2,
n_clusters_per_class=1, flip_y=0.09, class_sep = 1.1, random_state=r);
X1 = pd.DataFrame(X, columns=['Age', 'Income'])
y1 = pd.Series(y, name='Default')
df = pd.concat([X1, y1], axis=1)
plt.figure(figsize=(16, 10));
plt.grid(True);
ind_d = y==1
ind_p = y==0
plt.scatter(X[ind_d,0], X[ind_d,1], marker='o', s=200, label='Defaulted');
plt.scatter(X[ind_p,0], X[ind_p,1], marker='o', s=200, label="Paid", alpha=0.8);
plt.legend(fontsize=16);
plt.title("Default Dataset", fontsize=20);
plt.xlabel('Age (normalized)', fontsize=22);
plt.ylabel('Income (normalized)', fontsize=22);
plt.xticks(fontsize=18);
plt.yticks(fontsize=18);
plt.tight_layout();
plt.savefig('out/default-data.png');
X.shape
X[1:10,:]
y.shape
y[1:10]
df.head(15)
import pandas_profiling
pandas_profiling.ProfileReport(df, check_correlation=False)
from yellowbrick.target import FeatureCorrelation
visualizer = FeatureCorrelation(method='mutual_info-classification', sort=True)
visualizer.fit(X, y, random_state=0)
visualizer.poof()
import seaborn as sns
sns.set(style="ticks")
sns.pairplot(df, hue="Default");
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
from matplotlib.colors import ListedColormap
from sklearn.metrics import roc_curve, auc
# Adopted from: https://scikit-learn.org/stable/auto_examples/classification/plot_classifier_comparison.html
def plot_boundaries(X_train, X_test, y_train, y_test, clf, clf_name, ax, hide_ticks=True):
cm = plt.cm.RdBu
cm_bright = ListedColormap(['#FF0000', '#0000FF'])
X = np.concatenate((X_train, X_test), axis=0)
y = np.concatenate((y_train, y_test), axis=0)
x_min, x_max = X[:, 0].min() - .5, X[:, 0].max() + .5
y_min, y_max = X[:, 1].min() - .5, X[:, 1].max() + .5
xx, yy = np.meshgrid(np.arange(x_min, x_max, 0.02), np.arange(y_min, y_max, 0.02));
score = clf.score(X_test, y_test);
# Plot the decision boundary. For that, we will assign a color to each
# point in the mesh [x_min, x_max]x[y_min, y_max].
if hasattr(clf, "decision_function"):
Z = clf.decision_function(np.c_[xx.ravel(), yy.ravel()]);
else:
Z = clf.predict_proba(np.c_[xx.ravel(), yy.ravel()])[:, 1];
# Put the result into a color plot
Z = Z.reshape(xx.shape)
ax.contourf(xx, yy, Z, cmap=cm, alpha=.8);
# Plot the training points
ax.scatter(X_train[:, 0], X_train[:, 1], c=y_train, s=100, cmap=cm_bright, edgecolors='k');
# Plot the testing points
ax.scatter(X_test[:, 0], X_test[:, 1], c=y_test, s=50, cmap=cm_bright, edgecolors='k', alpha=0.6);
ax.set_xlim(xx.min(), xx.max());
ax.set_ylim(yy.min(), yy.max());
if hide_ticks:
ax.set_xticks(());
ax.set_yticks(());
else:
ax.tick_params(axis='both', which='major', labelsize=18)
#ax.yticks(fontsize=18);
ax.set_title(clf_name, fontsize=28);
ax.text(xx.max() - .3, yy.min() + .3, ('%.2f' % score).lstrip('0'), size=35, horizontalalignment='right');
ax.grid();
def plot_roc(clf, X_test, y_test, name, ax, show_thresholds=True):
y_pred_rf = clf.predict_proba(X_test)[:, 1]
fpr, tpr, thr = roc_curve(y_test, y_pred_rf)
ax.plot([0, 1], [0, 1], 'k--');
ax.plot(fpr, tpr, label='{}, AUC={:.2f}'.format(name, auc(fpr, tpr)));
ax.scatter(fpr, tpr);
if show_thresholds:
for i, th in enumerate(thr):
ax.text(x=fpr[i], y=tpr[i], s="{:.2f}".format(th), fontsize=14,
horizontalalignment='left', verticalalignment='top', color='black',
bbox=dict(facecolor='white', edgecolor='black', boxstyle='round,pad=0.1', alpha=0.1));
ax.set_xlabel('False positive rate', fontsize=18);
ax.set_ylabel('True positive rate', fontsize=18);
ax.tick_params(axis='both', which='major', labelsize=18);
ax.grid(True);
ax.set_title('ROC Curve', fontsize=18)
from sklearn.tree import DecisionTreeClassifier
clf = DecisionTreeClassifier(random_state=42, criterion="entropy",
min_samples_split=10, min_samples_leaf=10, max_depth=3, max_leaf_nodes=5)
clf.fit(X_train, y_train)
y_pred_dt = clf.predict(X_test)
feature_names = X1.columns
class_names = [str(x) for x in clf.classes_]
clf.predict_proba([[2, 2]])
clf.predict([[2, 2]])
Surpisingly, sci-kit learn does not have a function to print the decision tree in text format. (It does have a way to graphical render the three, which we'll do later.) For now, we'll just print a few stats about the tree.
print(clf.tree_.node_count)
print(clf.tree_.impurity)
print(clf.tree_.children_left)
print(clf.tree_.threshold)
from pandas_ml import ConfusionMatrix
print(ConfusionMatrix(y_test, y_pred_dt))
from sklearn.metrics import classification_report
print(classification_report(y_test, y_pred_dt, target_names=class_names))
from sklearn.metrics import accuracy_score, cohen_kappa_score, f1_score, log_loss
print("Accuracy = {:.2f}".format(accuracy_score(y_test, y_pred_dt)))
print("Kappa = {:.2f}".format(cohen_kappa_score(y_test, y_pred_dt)))
print("F1 Score = {:.2f}".format(f1_score(y_test, y_pred_dt)))
print("Log Loss = {:.2f}".format(log_loss(y_test, y_pred_dt)))
from yellowbrick.classifier import ClassificationReport
# Instantiate the classification model and visualizer
visualizer = ClassificationReport(clf, classes=class_names, support=True)
visualizer.fit(X_train, y_train) # Fit the visualizer and the model
visualizer.score(X_test, y_test) # Evaluate the model on the test data
g = visualizer.poof() # Draw/show/poof the data
from yellowbrick.classifier import ClassPredictionError
visualizer = ClassPredictionError(clf, classes=class_names)
visualizer.fit(X_train, y_train)
visualizer.score(X_test, y_test)
g = visualizer.poof()
from yellowbrick.classifier import ROCAUC
visualizer = ROCAUC(clf, classes=class_names)
visualizer.fit(X_train, y_train) # Fit the training data to the visualizer
visualizer.score(X_test, y_test) # Evaluate the model on the test data
g = visualizer.poof() # Draw/show/poof the data
plt.style.use('default');
figure = plt.figure(figsize=(10, 6));
ax = plt.subplot(1, 1, 1);
plot_roc(clf, X_test, y_test, "Decision Tree", ax)
plt.legend(loc='lower right', fontsize=18);
plt.tight_layout();
plt.savefig('out/default-dt-roc.png');
from yellowbrick.classifier import PrecisionRecallCurve
viz = PrecisionRecallCurve(clf);
viz.fit(X_train, y_train);
viz.score(X_test, y_test);
viz.poof();
from yellowbrick.classifier import DiscriminationThreshold
visualizer = DiscriminationThreshold(clf)
visualizer.fit(X, y) # Fit the training data to the visualizer
visualizer.poof() # Draw/show/poof the data
from sklearn.tree import export_graphviz
import graphviz
base='out/default_dt_graph'
dot = base + '.dot'
png = base + '.png'
dot_data = export_graphviz(clf, out_file=dot, feature_names=feature_names, class_names=class_names,
filled=True, rounded=True, special_characters=True,
leaves_parallel=False, proportion=False)
graph = graphviz.Source(dot_data)
from subprocess import call
call(['dot', '-Tpng', dot, '-o', png, '-Gdpi=600'])
# Display in python
import matplotlib.pyplot as plt
plt.figure(figsize = (24, 28));
plt.imshow(plt.imread(png));
plt.axis('off');
plt.show();
import importlib
spec = importlib.util.find_spec("dtreeviz")
dtreeviz_found = spec is not None
if dtreeviz_found:
from dtreeviz.trees import *
viz = dtreeviz(clf, X, y, target_name='Default', feature_names=feature_names, class_names=class_names, orientation ='LR');
viz.save('out/default_dt_dtreeviz1.svg')
viz
else:
print("dtreeviz not installed on this system.")
if dtreeviz_found:
viz = dtreeviz(clf, X, y, target_name='Default', feature_names=feature_names, class_names=class_names, orientation ='TD', fancy=False);
viz.save('out/default_dt_dtreeviz2.svg')
viz
else:
print("dtreeviz not installed on this system.")
if dtreeviz_found:
# Select a random sample
x = X_train[13,:]
print(x)
print(y_train[13])
viz = dtreeviz(clf, X_train, y_train, target_name='default', feature_names=feature_names, class_names=class_names, orientation ='LR', X=x);
viz
else:
print("dtreeviz not installed on this system.")
figure = plt.figure(figsize=(10, 10));
ax = plt.subplot(1, 1, 1);
plot_boundaries(X_train, X_test, y_train, y_test, clf, "Decision Tree", ax, hide_ticks=False)
ax.set_xlabel("Age", fontsize=22)
ax.set_ylabel("Income", fontsize=22)
plt.tight_layout();
plt.savefig('out/default-dt-5-boundaries.png');
from sklearn.model_selection import GridSearchCV
parameters = {'kernel':('linear', 'rbf'), 'C':[1, 10]}
treeclf = DecisionTreeClassifier(splitter='best', presort=True, class_weight=None, random_state=42)
parameters = {'criterion':('gini', 'entropy'), 'max_depth':[2, 4, 6, 8, 10], 'min_samples_split':[2, 10, 50], 'min_samples_leaf':[1, 5, 10],
'max_features':[None, 'auto'], 'max_leaf_nodes':[None, 5, 10, 50], 'min_impurity_decrease':[0, 0.1, 0.2]}
cv_clf = GridSearchCV(treeclf, parameters, scoring='roc_auc', cv=5, return_train_score=True)
%time cv_clf.fit(X, y)
cv_clf.best_params_
cv_clf.best_score_
cv_clf.best_estimator_
figure = plt.figure(figsize=(10, 10));
ax = plt.subplot(1, 1, 1);
plot_boundaries(X_train, X_test, y_train, y_test, cv_clf.best_estimator_, "Decision Tree", ax, hide_ticks=False)
ax.set_xlabel("Age", fontsize=22)
ax.set_ylabel("Income", fontsize=22)
plt.tight_layout();
plt.savefig('out/default-dt-best-boundaries.png');
if dtreeviz_found:
viz = dtreeviz(cv_clf.best_estimator_, X, y, target_name='Default', feature_names=feature_names, class_names=class_names, orientation ='TD', fancy=False);
viz
from yellowbrick.model_selection import ValidationCurve
viz = ValidationCurve(DecisionTreeClassifier(), param_name="max_depth", param_range=np.arange(1, 11), cv=10, scoring="roc_auc")
viz.fit(X, y)
viz.poof(outpath='out/dt_validation_curve.png')
viz.poof()
from sklearn.naive_bayes import GaussianNB
gnb = GaussianNB()
gnb = gnb.fit(X_train, y_train)
gnb
y_pred_gnb = gnb.predict(X_test)
from sklearn.metrics import confusion_matrix
confusion_matrix(y_test, y_pred_gnb)
from sklearn.metrics import classification_report
gnb.theta_ # Mean of each feature per class
gnb.sigma_ # Variance of each feature per class
print(ConfusionMatrix(y_test, y_pred_gnb))
print(classification_report(y_test, y_pred_gnb, target_names=class_names))
print("Accuracy = {:.2f}".format(accuracy_score(y_test, y_pred_gnb)))
print("Kappa = {:.2f}".format(cohen_kappa_score(y_test, y_pred_gnb)))
print("F1 Score = {:.2f}".format(f1_score(y_test, y_pred_gnb)))
print("Log Loss = {:.2f}".format(log_loss(y_test, y_pred_gnb)))
from yellowbrick.classifier import DiscriminationThreshold
visualizer = DiscriminationThreshold(gnb)
visualizer.fit(X, y) # Fit the training data to the visualizer
visualizer.poof() # Draw/show/poof the data
plt.style.use('default');
figure = plt.figure(figsize=(10, 6));
ax = plt.subplot(1, 1, 1);
plot_roc(gnb, X_test, y_test, "GNB", ax)
plt.legend(loc='lower right', fontsize=18);
plt.tight_layout();
plt.savefig('out/default-gnb-roc.png');
figure = plt.figure(figsize=(10, 10));
ax = plt.subplot(1, 1, 1);
plot_boundaries(X_train, X_test, y_train, y_test, gnb, "Gaussian NB", ax, hide_ticks=False)
ax.set_xlabel("Age", fontsize=22)
ax.set_ylabel("Income", fontsize=22)
plt.tight_layout();
plt.savefig('out/default-gnb-boundaries.png');
from sklearn.neighbors import KNeighborsClassifier
knn_clf = KNeighborsClassifier(n_neighbors=3)
knn_clf.fit(X_train, y_train)
y_pred_knn = knn_clf.predict(X_test)
knn_clf.effective_metric_
knn_clf.effective_metric_params_
print(ConfusionMatrix(y_test, y_pred_knn))
print(classification_report(y_test, y_pred_knn, target_names=class_names))
print("Accuracy = {:.2f}".format(accuracy_score(y_test, y_pred_knn)))
print("Kappa = {:.2f}".format(cohen_kappa_score(y_test, y_pred_knn)))
print("F1 Score = {:.2f}".format(f1_score(y_test, y_pred_knn)))
print("Log Loss = {:.2f}".format(log_loss(y_test, y_pred_knn)))
from yellowbrick.classifier import DiscriminationThreshold
visualizer = DiscriminationThreshold(knn_clf)
visualizer.fit(X, y) # Fit the training data to the visualizer
visualizer.poof() # Draw/show/poof the data
plt.style.use('default');
figure = plt.figure(figsize=(10, 6));
ax = plt.subplot(1, 1, 1);
plot_roc(knn_clf, X_test, y_test, "GNB", ax)
plt.legend(loc='lower right', fontsize=18);
plt.tight_layout();
plt.savefig('out/default-knn-roc.png');
figure = plt.figure(figsize=(10, 10));
ax = plt.subplot(1, 1, 1);
plot_boundaries(X_train, X_test, y_train, y_test, knn_clf, "KNN (3)", ax, hide_ticks=False)
ax.set_xlabel("Age", fontsize=22)
ax.set_ylabel("Income", fontsize=22)
plt.tight_layout();
plt.savefig('out/default-knn-boundaries.png');
rng = np.random.RandomState(2)
figure = plt.figure(figsize=(27, 27));
i = 1
ks = np.arange(1, 13)
for k in ks:
ax = plt.subplot(4, 3, i);
clf_tmp = KNeighborsClassifier(n_neighbors=k)
clf_tmp.fit(X_train, y_train);
plot_boundaries(X_train, X_test, y_train, y_test, clf_tmp, "KNN (k={:d})".format(k), ax, hide_ticks=True);
i += 1
plt.tight_layout();
plt.savefig('out/default-knn-all.png');
from yellowbrick.model_selection import ValidationCurve
viz = ValidationCurve(KNeighborsClassifier(), param_name="n_neighbors", param_range=np.arange(1, 25), cv=5, scoring="roc_auc")
viz.fit(X, y)
viz.poof(outpath='out/default-knn-validation.png')
viz.poof()
from sklearn.svm import SVC
svm_clf = SVC(kernel="linear", C=0.025)
svm_clf.fit(X_train, y_train)
y_pred_svm = svm_clf.predict(X_test)
svm_clf.n_support_
svm_clf.support_vectors_
svm_clf.dual_coef_
svm_clf.intercept_
plt.figure(figsize=(16, 10));
plt.grid(True);
ind_d = y_train==1
ind_p = y_train==0
plt.scatter(X_train[ind_d,0], X_train[ind_d,1], marker='o', s=200, label='Defaulted');
plt.scatter(X_train[ind_p,0], X_train[ind_p,1], marker='o', s=200, label="Paid", alpha=0.8);
# plot the decision function
ax = plt.gca()
xlim = ax.get_xlim()
ylim = ax.get_ylim()
# create grid to evaluate model
xx = np.linspace(xlim[0], xlim[1], 30)
yy = np.linspace(ylim[0], ylim[1], 30)
YY, XX = np.meshgrid(yy, xx)
xy = np.vstack([XX.ravel(), YY.ravel()]).T
Z = svm_clf.decision_function(xy).reshape(XX.shape)
# plot decision boundary and margins
ax.contour(XX, YY, Z, colors='k', levels=[-1, 0, 1], alpha=0.5, linestyles=['--', '-', '--'])
# plot support vectors
plt.scatter(svm_clf.support_vectors_[:, 0], svm_clf.support_vectors_[:, 1], s=200, linewidth=3, facecolors='none', edgecolors='k')
plt.legend(fontsize=16);
plt.title("SVM (Linear)", fontsize=20);
plt.xlabel('Age (normalized)', fontsize=22);
plt.ylabel('Income (normalized)', fontsize=22);
plt.xticks(fontsize=18);
plt.yticks(fontsize=18);
plt.tight_layout();
plt.savefig('out/default-svm-support-vectors.png');
print(ConfusionMatrix(y_test, y_pred_svm))
print(classification_report(y_test, y_pred_svm, target_names=class_names))
print("Accuracy = {:.2f}".format(accuracy_score(y_test, y_pred_svm)))
print("Kappa = {:.2f}".format(cohen_kappa_score(y_test, y_pred_svm)))
print("F1 Score = {:.2f}".format(f1_score(y_test, y_pred_svm)))
print("Log Loss = {:.2f}".format(log_loss(y_test, y_pred_svm)))
figure = plt.figure(figsize=(10, 10));
ax = plt.subplot(1, 1, 1);
plot_boundaries(X_train, X_test, y_train, y_test, svm_clf, "SVM (Linear)", ax, hide_ticks=False)
ax.set_xlabel("Age", fontsize=22)
ax.set_ylabel("Income", fontsize=22)
plt.tight_layout();
plt.savefig('out/default-svm-linear-boundaries.png');
names = ["Linear C=0.0025", "Linear C=0.25", "Linear C=25"]
classifiers = [
SVC(kernel="linear", C=0.0025),
SVC(kernel="linear", C=0.25),
SVC(kernel="linear", C=25),
]
rng = np.random.RandomState(2)
figure = plt.figure(figsize=(27, 10));
i = 1
# iterate over classifiers
for name, clf_tmp in zip(names, classifiers):
ax = plt.subplot(1, 3, i);
clf_tmp.fit(X_train, y_train);
plot_boundaries(X_train, X_test, y_train, y_test, clf_tmp, name, ax, hide_ticks=True);
i += 1
plt.tight_layout();
plt.savefig('out/default-svm-linear-all.png');
names = ["Poly 2", "Poly 3", "Poly 4"]
classifiers = [
SVC(kernel="poly", degree=2, C=0.25),
SVC(kernel="poly", degree=3, C=1),
SVC(kernel="poly", degree=4, C=1),
]
rng = np.random.RandomState(2)
figure = plt.figure(figsize=(27, 10));
i = 1
# iterate over classifiers
for name, clf_tmp in zip(names, classifiers):
ax = plt.subplot(1, 3, i);
clf_tmp.fit(X_train, y_train);
plot_boundaries(X_train, X_test, y_train, y_test, clf_tmp, name, ax, hide_ticks=True);
i += 1
plt.tight_layout();
plt.savefig('out/default-svm-poly-all.png');
names = ["RBF G=0.05", "RBF G=0.5", "RBF G=5.0"]
classifiers = [
SVC(kernel="rbf", gamma=0.05, C=1),
SVC(kernel="rbf", gamma=0.5, C=1),
SVC(kernel="rbf", gamma=5.0, C=1),
]
rng = np.random.RandomState(2)
figure = plt.figure(figsize=(27, 10));
i = 1
# iterate over classifiers
for name, clf_tmp in zip(names, classifiers):
ax = plt.subplot(1, 3, i);
clf_tmp.fit(X_train, y_train);
plot_boundaries(X_train, X_test, y_train, y_test, clf_tmp, name, ax, hide_ticks=True);
i += 1
plt.tight_layout();
plt.savefig('out/default-svm-rbf-all.png');
from sklearn.neural_network import MLPClassifier
nn_clf = MLPClassifier(solver='lbfgs', activation='relu', alpha=1e-3,
hidden_layer_sizes=(3), random_state=1, verbose=True)
nn_clf.fit(X_train, y_train)
y_pred_nn = nn_clf.predict(X_test)
nn_clf.loss_
nn_clf.n_layers_
w = nn_clf.coefs_ # The ith element in the list represents the weight matrix corresponding to layer i.
w
b = nn_clf.intercepts_ # The ith element in the list represents the bias vector corresponding to layer i + 1.
b
nn_clf.out_activation_
nn_clf.predict_proba([[0.5, 1]])
def relu_af(x):
return max(0, x)
x = [0.5, 1]
layer_id=0
h0 = (x[0] * w[layer_id][0][0]) + ((x[1] * w[layer_id][1][0])) + (1 * b[layer_id][0])
print(h0)
print(relu_af(h0))
h1 = (x[0] * w[layer_id][0][1]) + ((x[1] * w[layer_id][1][1])) + (1 * b[layer_id][1])
print(h1)
print(relu_af(h1))
h2 = (x[0] * w[layer_id][0][2]) + ((x[1] * w[layer_id][1][2])) + (1 * b[layer_id][2])
print(h2)
print(relu_af(h2))
layer_id=1
o = (h0 * w[layer_id][0][0]) + ((h1 * w[layer_id][1][0])) + ((h2 * w[layer_id][2][0])) + (1 * b[layer_id][0])
print(o)
print(relu_af(o))
print(ConfusionMatrix(y_test, y_pred_nn))
print(classification_report(y_test, y_pred_nn, target_names=class_names))
print("Accuracy = {:.2f}".format(accuracy_score(y_test, y_pred_nn)))
print("Kappa = {:.2f}".format(cohen_kappa_score(y_test, y_pred_nn)))
print("F1 Score = {:.2f}".format(f1_score(y_test, y_pred_nn)))
print("Log Loss = {:.2f}".format(log_loss(y_test, y_pred_nn)))
figure = plt.figure(figsize=(10, 10));
ax = plt.subplot(1, 1, 1);
plot_boundaries(X_train, X_test, y_train, y_test, nn_clf, "NN MLP", ax, hide_ticks=False)
ax.set_xlabel("Age", fontsize=22)
ax.set_ylabel("Income", fontsize=22)
plt.tight_layout();
plt.savefig('out/default-nn-mlp-boundaries.png');
rng = np.random.RandomState(2)
alpha=0.001
archs = [(5), (5, 5), (10, 2)]
afs = ['identity', 'logistic', 'tanh', 'relu']
j = 1
for arch in archs:
figure = plt.figure(figsize=(27, 10));
i = 1
for af in afs:
ax = plt.subplot(1, 4, i);
clf_tmp = MLPClassifier(solver='lbfgs', activation=af, alpha=alpha, hidden_layer_sizes=arch, random_state=1);
clf_tmp = clf_tmp.fit(X_train, y_train);
plot_boundaries(X_train, X_test, y_train, y_test, clf_tmp, "NN (af={}, arch={})".format(af, arch, alpha), ax, hide_ticks=True);
i += 1
plt.tight_layout();
plt.savefig('out/default-nn-arch-{:d}-af.png'.format(j));
j += 1
rng = np.random.RandomState(2)
figure = plt.figure(figsize=(27, 27));
i = 1
alphas = 10.0 ** -np.arange(1, 12)
print(alphas)
for alpha in alphas:
ax = plt.subplot(4, 3, i);
clf_tmp = MLPClassifier(solver='lbfgs', activation='relu', alpha=alpha, hidden_layer_sizes=(10, 2), random_state=1)
clf_tmp.fit(X_train, y_train);
plot_boundaries(X_train, X_test, y_train, y_test, clf_tmp, "NN (alpha={:.5f})".format(alpha), ax, hide_ticks=True);
i += 1
plt.tight_layout();
plt.savefig('out/default-nn-all.png');
from keras.models import Sequential
from keras.layers import Dense
from keras.utils import np_utils
model = Sequential()
model.add(Dense(3, input_dim=2, activation="relu"))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
%time history = model.fit(X, y, validation_split=0.20, epochs=100, batch_size=10, verbose=0)
# list all data in history
print(history.history.keys())
figure = plt.figure(figsize=(12, 8));
plt.plot(history.history['acc']);
plt.plot(history.history['val_acc']);
plt.title('Keras Model Accuracy', fontsize=22);
plt.ylabel('accuracy', fontsize=18);
plt.xlabel('epoch', fontsize=18);
plt.xticks(fontsize=14);
plt.yticks(fontsize=14);
plt.ylim(bottom=0.85);
plt.legend(['train', 'test'], loc='upper left', fontsize=14);
plt.grid();
plt.tight_layout();
plt.savefig('out/default-keras-accuracy.png');
# calculate predictions
predictions = model.predict(X_test)
# round predictions
y_pred_keras = [round(x[0]) for x in predictions]
print(ConfusionMatrix(y_test, y_pred_keras))
#from keras.utils import plot_model
#plot_model(model, to_file='out/default-keras-model.png', show_shapes=True, show_layer_names=True)
#from ann_visualizer.visualize import ann_viz
#ann_viz(model, view=True, filename='out/default-keras-ann.gv', title="Default Dataset")
# Inspired by: https://scikit-learn.org/stable/auto_examples/classification/plot_classifier_comparison.html
import numpy as np
import matplotlib.pyplot as plt
from matplotlib.colors import ListedColormap
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.datasets import make_moons, make_circles, make_classification
from sklearn.neural_network import MLPClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC
from sklearn.gaussian_process import GaussianProcessClassifier
from sklearn.gaussian_process.kernels import RBF
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis
names = ["KNN 3", "Linear SVM", "RBF SVM", "Gaussian Process",
"Decision Tree", "Random Forest", "Neural Net", "AdaBoost",
"Naive Bayes"]
classifiers = [
KNeighborsClassifier(3),
SVC(kernel="linear", C=0.025),
SVC(gamma=2, C=1),
GaussianProcessClassifier(1.0 * RBF(1.0)),
DecisionTreeClassifier(max_depth=5),
RandomForestClassifier(max_depth=5, n_estimators=10, max_features=1),
MLPClassifier(alpha=1),
AdaBoostClassifier(),
GaussianNB()]
rng = np.random.RandomState(2)
figure = plt.figure(figsize=(27, 27));
i = 1
# iterate over classifiers
for name, clf_tmp in zip(names, classifiers):
ax = plt.subplot(3, 3, i);
clf_tmp.fit(X_train, y_train);
plot_boundaries(X_train, X_test, y_train, y_test, clf_tmp, name, ax, hide_ticks=False);
i += 1
plt.tight_layout();
plt.savefig('out/default-all.png');